package tk.mybatis.springboot.spider.processor;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import tk.mybatis.springboot.spider.entity.ErrorEntity;
import tk.mybatis.springboot.spider.entity.ImowProductEntity;
import tk.mybatis.springboot.spider.entity.TestProduct;
import tk.mybatis.springboot.spider.utils.UrlConfig;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;


/**
 * Created by xlp on 2018/1/17.
 */
public class ImowPageProcessor implements PageProcessor {
    private Site site = Site.me()
            .setRetryTimes(3) //这个重试一般用处不大
            .setTimeOut(30000)//在使用代理的情况下,这个需要设置,可以考虑调大线程数目
            .setSleepTime(5000)//使用代理了之后,代理会通过切换IP来防止反扒。同时,使用代理本身qps降低了,所以这个可以小一些
            .setCycleRetryTimes(3)//这个重试会换IP重试,是setRetryTimes的上一层的重试,不要怕三次重试解决一切问题。。
            .setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36")
            .setUseGzip(true);

    /**
     * @param page
     */
    @Override
    public void process(Page page) {
        /**
         * 首页-获取分类链接
         */
        if (page.getUrl().regex(UrlConfig.IMOW_DOMAIN).match()) {
            try {
                print("**********阿母工业首页**********\n", "");
                String html = page.getHtml().toString();
                Document doc = Jsoup.parse(html);
//            直接获取三级分类链接
                Element cateUlE = doc.getElementsByClass("cate-ul").first();
                //每一个item：cate-conts cate-popbox
                List<Element> popBoxs = cateUlE.getElementsByClass("cate-popbox");
                List<Element> cateConts = cateUlE.getElementsByClass("cate-conts");
                //
                for (int i = 0; i < cateConts.size(); i++) {

                    Element cateCont = cateConts.get(i).getElementsByTag("a").first();
                    String cate1Link = cateCont.attr("href");
                    String cate1Name = cateCont.text();

                    print("一级分类链接：", cate1Link);
                    print("一级分类名称：", cate1Name);
                    List<Element> cates = popBoxs.get(i).getElementsByTag("dl");
                    for (int j = 0; j < cates.size(); j++) {

                        Element cate2E = cates.get(j).select(" > dt > a").first();
                        String cate2Link = cate2E.attr("href");
                        String cate2Name = cate2E.text();

                        print("二级分类链接：", cate2Link);
                        print("二级分类名称：", cate2Name);
                        List<Element> cate3E = cates.get(j).select("> dd > ul > li > a");
                        for (int k = 0; k < cate3E.size(); k++) {

                            String cate3Link = cate3E.get(k).attr("href");
                            String cate3Name = cate3E.get(k).text();

                            print("三级分类链接：", cate3Link);
                            print("三级分类名称：", cate3Name);
                            page.addTargetRequest(cate3Link);
                        }
                        print("=======================", "");
                    }
                    print("**********", "**********");
                }
            } catch (Exception e) {
                ErrorEntity errorEntity = new ErrorEntity();
                errorEntity.setErrorLink(page.getUrl().toString());
                errorEntity.setError(e.getCause());
                errorEntity.setErrorType("首页-获取分类链接");
                page.putField("errorLink", errorEntity);
                e.printStackTrace();

            }
        }
        /**
         * 三级分类页面-获取商品列表
         */
        if (page.getUrl().regex(UrlConfig.IMOW_CATEGORY3).match()) {
            try {
                String html = page.getHtml().toString();
                Document doc = Jsoup.parse(html);
                Element products = doc.getElementsByClass("p-list").first();
                List<Element> detailBoxs = products.getElementsByClass("goods-item-box");
                if (detailBoxs.size() > 0) {
                    for (Element detailBox : detailBoxs) {
                        Element detailLinkE = detailBox.getElementsByTag("a").first();
                        String detailLink = detailLinkE.attr("href");
                        println("##########详情页链接##########", detailLink);
//                    page.addTargetRequest(detailLink);
                    }
                } else {
                    ErrorEntity errorEntity = new ErrorEntity();
                    errorEntity.setErrorLink(page.getUrl().toString());
                    errorEntity.setError("商品列表页无商品");
                    errorEntity.setErrorType("商品列表页");
                    page.putField("errorLink", errorEntity);
                    page.setSkip(true);
                }
            } catch (Exception e) {
                ErrorEntity errorEntity = new ErrorEntity();
                errorEntity.setErrorLink(page.getUrl().toString());
                errorEntity.setError(e.getCause());
                errorEntity.setErrorType("三级分类页面-获取商品列表");
                page.putField("errorLink", errorEntity);
                e.printStackTrace();
            }

        }
        /**
         * 商品详情页
         * http://www.imow.cn/product/181694313233475.html
         */
        if (page.getUrl().regex(UrlConfig.IMOW_DETAIL).match()) {
            try {
                System.out.println("详情页");
                String html = page.getHtml().toString();
                Document doc = Jsoup.parse(html);
                ImowProductEntity product = new ImowProductEntity();
                Element navE = doc.getElementById("nav");
                Element headImgE = doc.getElementsByClass("pt-header-imgleft").first();
                System.out.println(headImgE.toString());
                Element infoAreaE = doc.getElementsByClass("anchorcontent").first();
                List<Element> navs = navE.getElementsByTag("a");
                List<String> navList = new ArrayList<String>();
                for (int i = 0; i < navs.size(); i++) {
                    navList.add(navs.get(i).text());
                }
                Element currentNav = navE.getElementsByTag("span").get(1);
                navList.add(navList.size(), currentNav.text());


                Element productsTable = doc.select(".select").first();
                List<Element> productTrs = productsTable.getElementsByTag("tr");
                print("sku个数：", "" + (productTrs.size() - 1));
                for (int i = 1; i < productTrs.size(); i++) {
                    List<Element> productTds = productTrs.get(i).getElementsByTag("td");
                    setCatesByNav(product, navList);
                    setProductInfo(product, productTrs, productTds);
                    String headImg = headImgE.getElementsByTag("img").first().attr("data-original");
                    product.setProductImg(headImg);
                    println("==========商品小图==========", product.getProductImg());
                    product.setProductInfoArea1(infoAreaE.toString());
                    println("==========商品详情图片==========", product.getProductInfoArea1().toString());
                    System.out.println("");
                    page.putField("productEntity", product);
                }
            } catch (Exception e) {
                System.out.println("当前错误链接：" + page.getUrl());
                ErrorEntity errorEntity = new ErrorEntity();
                errorEntity.setErrorLink(page.getUrl().toString());
                errorEntity.setError(e.getCause());
                errorEntity.setErrorType("商品详情页");
                page.putField("errorLink", errorEntity);
                e.printStackTrace();

            }
        }

    }

    @Override
    public Site getSite() {
        return this.site;
    }

    /**
     * 设置分类
     */
    public void setCatesByNav(ImowProductEntity product, List<String> navList) {
        switch (navList.size()) {
            case 2:
                product.setProductCate1(navList.get(0));
                product.setProductCate2(navList.get(1));
                product.setProductName(navList.get(1));

                break;
            case 3:
                product.setProductCate1(navList.get(0));
                product.setProductCate2(navList.get(1));
                product.setProductCate3(navList.get(2));
                product.setProductName(navList.get(2));

                break;
            case 4:
                product.setProductCate1(navList.get(0));
                product.setProductCate2(navList.get(1));
                product.setProductCate3(navList.get(2));
                product.setProductCate4(navList.get(3));
                product.setProductName(navList.get(3));


                break;
            case 5:
                product.setProductCate1(navList.get(0));
                product.setProductCate2(navList.get(1));
                product.setProductCate3(navList.get(2));
                product.setProductCate4(navList.get(3));
                println("==========四级分类==========", product.getProductCate4());
                product.setProductCate5(navList.get(4));
                println("==========五级分类==========", product.getProductCate5());

                product.setProductName(navList.get(4));

                break;
            case 6:
                product.setProductCate1(navList.get(0));
                product.setProductCate2(navList.get(1));
                product.setProductCate3(navList.get(2));
                product.setProductCate4(navList.get(3));
                println("==========四级分类==========", product.getProductCate4());
                product.setProductCate5(navList.get(4));
                println("==========五级分类==========", product.getProductCate5());
                product.setProductName(navList.get(4));
                break;
        }

        println("==========商品名称==========", product.getProductName());
        println("==========一级分类==========", product.getProductCate1());
        println("==========二级分类==========", product.getProductCate2());
        println("==========三级分类==========", product.getProductCate3());


    }

    /**
     * 设置商品信息
     */
    public void setProductInfo(ImowProductEntity product, List<Element> trList, List<Element> tdList) {
        int priceIndex = tdList.size() - 2;

        if (tdList.size() >= 2 && null != tdList.get(1)) {
            product.setProductSku(tdList.get(1).text());
        }
        if (tdList.size() >= 3 && null != tdList.get(2)) {
            product.setProductInfo1(tdList.get(2).text());
        }
        if (tdList.size() >= 4 && null != tdList.get(3)) {
            product.setProductInfo2(tdList.get(3).text());
        }
        if (tdList.size() >= 5 && null != tdList.get(4)) {
            product.setProductInfo3(tdList.get(4).text());
        }
        if (tdList.size() >= 6 && null != tdList.get(5)) {
            product.setProductInfo4(tdList.get(5).text());
        }
        if (tdList.size() >= 7 && null != tdList.get(6)) {
            product.setProductInfo5(tdList.get(6).text());
        }
        if (tdList.size() >= 8 && null != tdList.get(7)) {
            product.setProductInfo6(tdList.get(7).text());
        }
        if (tdList.size() >= 9 && null != tdList.get(8)) {
            product.setProductInfo7(tdList.get(8).text());
        }


        if (null != tdList.get(priceIndex)) {
            Element priceE = tdList.get(priceIndex).getElementsByClass("price").first();
            if (null != priceE)
                if (trList.get(0).text().contains("价格") || trList.get(0).text().contains("元")) {
                    String price = tdList.get(priceIndex).text();
                    String unit = price.substring(price.indexOf("元/") + 2);
                    product.setProductPrice(new BigDecimal(priceE.text()));
                    product.setProductUnit(unit);
                } else {
                    product.setProductInfo7(tdList.get(priceIndex).text());
                }
        }
        println("==========商品信息1==========", product.getProductInfo1());
        println("==========商品信息2==========", product.getProductInfo2());
        println("==========商品信息3==========", product.getProductInfo3());
        println("==========商品信息4==========", product.getProductInfo4());
        println("==========商品信息5==========", product.getProductInfo5());
        println("==========商品信息6==========", product.getProductInfo6());
        println("==========商品信息7==========", product.getProductInfo7());
        println("==========价格==========", product.getProductPrice() + "");
        println("==========单位==========", product.getProductUnit());
    }

    /**
     * 反射机制 动态设置属性的值
     *
     * @param testProductEntity
     * @param list
     */
    public void setCates(TestProduct testProductEntity, List<String> list) {
        Field[] field = testProductEntity.getClass().getDeclaredFields(); // 获取实体类的所有属性(不包括父类)
        Map<String, String> map1 = new HashMap<String, String>();

        for (int i = 0; i < list.size(); i++) {
            String key = field[i].getName();
            map1.put(key, list.get(i));
        }

        try {
            for (int j = 0; j < list.size(); j++) {
                String name = field[j].getName(); //获取属性名
                String oldName = name;
                System.out.println("field old name:" + name);
                // 把属性首字母变大写
                name = name.substring(0, 1).toUpperCase() + name.substring(1);
                String type = field[j].getGenericType().toString(); //获取属性的类型
                if (type.equals("class java.lang.String")) { // 如果type是类类型,则class后面跟类名
                    // 根据属性名，创建set方法
                    Method m = testProductEntity.getClass().getMethod("set" + name, String.class);
                    m.invoke(testProductEntity, map1.get(oldName));
                }
            }
            for (Field field2 : field) {//输出属性值，方法1
                String name = field2.getName();
                field2.setAccessible(true);
                System.out.println("方法1输出" + name + " value:" + field2.get(testProductEntity));
            }
            for (Field field2 : field) {//输出属性值，方法2
                String name = field2.getName();
                name = name.substring(0, 1).toUpperCase() + name.substring(1);
                Method m = testProductEntity.getClass().getMethod("get" + name);
                String val = (String) m.invoke(testProductEntity);
                System.out.println("方法2输出" + name + " value:" + val);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * 输出
     */
    public void println(String a, String b) {
        System.out.println(a);
        System.out.println(b);
    }

    /**
     * 同行输出
     */
    public void print(String a, String b) {
        System.out.println(a + "\t" + b);
    }

    /**
     * 打包需要注释掉main()
     */
//    public static void main(String[] args) {
//        Spider.create(new ImowPageProcessor())
//                .setDownloader(new DungProxyDownloader())
//                .thread(50)
//                .addUrl("http://www.imow.cn")
//                .runAsync();
//    }
}
